package info.galudisu.spark_learning

import org.apache.spark.{SparkConf, SparkContext}

/**
  * @author galudisu
  */
object Main {

  def main(args: Array[String]): Unit = {

    if (args.length <= 3) {
      System.err.println("Usage: WordCount <input> <output>")
      System.exit(1)
    }

    // Parameters
    val input = args(0)
    val output = args(1)
    // Create SparkContext
    val conf = new SparkConf().setAppName("myWordCount")
    val sc = new SparkContext(conf)
    // Read RDD
    val lines = sc.textFile(s"file://$input")
    // Calculate result
    val resultRdd = lines.flatMap(_.split(" ")).map((_, 1)).reduceByKey(_ + _)
    // Persist result
    resultRdd.saveAsTextFile(s"file://$output")
    // Stop
    sc.stop()

  }
}
